#include "crawler.h"

int crawler(const char *url, const char *path, const char *filename, char *mode)
{
    struct sockaddr_in servaddr;
	int sockfd;
	struct hostent *hptr;
	char **pptr;
	char hostip[INET_ADDRSTRLEN];

	bzero(&servaddr, sizeof(servaddr));
	servaddr.sin_family = AF_INET;
	servaddr.sin_port   = htons(80);

	sockfd = Socket(AF_INET, SOCK_STREAM, 0);

	if ( (hptr = gethostbyname(url)) == NULL) 
	{
		err_msg("gethostbyname error for host: %s: %s",	url, hstrerror(h_errno));
	}		

	pptr = hptr->h_addr_list;
	servaddr.sin_addr = *((struct in_addr*)*pptr);
	Inet_ntop(hptr->h_addrtype, *pptr, hostip, sizeof(hostip));

	Connect(sockfd, (struct sockaddr*)&servaddr, sizeof(servaddr));

	char *request = (char*)malloc(1024*sizeof(char));

	if(!strcmp(mode, "ab+"))
	{
		sprintf(request, "GET http://%s%s HTTP/1.1\r\nAccept: image/jpg, image/png, image/gif, */*\r\nHost:%s\r\nConnection:Close\r\n\r\n", url, path, hostip);
	}
	else
	{
		Inet_ntop(hptr->h_addrtype, *pptr, hostip, sizeof(hostip));
		sprintf(request, "GET %s HTTP/1.1\r\nAccept: text/html, */*\r\nAccept-Encoding: gzip\r\nHost: %s\r\nConnection: Close\r\n\r\n", path, hostip);
	}

	char *response = (char *)malloc(RESPONSE_SIZE);

	int send = write(sockfd, request, strlen(request));
	int length_read = read(sockfd, response, RESPONSE_SIZE);

    printf("request:\n%s", request);
    free(request);
    printf("response:\n%s\n", response);
    fflush(stdout);

    char *len_ptr = strstr(response, "Content-Length:");  
    FILE *fp = fopen(filename, "w"); //clear contents of the file
    fclose(fp);
    fp = fopen(filename, mode);

    if(len_ptr)
    {
    	len_ptr += strlen("Content-Length:");

    	int length_response = atoi(len_ptr);	
//printf("length_response = %d\n", length_response);

    	char * content = strstr(response, "\r\n\r\n"); //截获返回头部，以\r\n\r\n为标识
    	int length_head = content-response+4;

    	if(!strcmp(mode, "ab+"))
    	{
    		fwrite(response+length_head, sizeof(char), length_read-length_head, fp);   		
    	}
    	else
    	{
    		ungzip(response+length_head, length_read-length_head, fp);
    		//fwrite(response+length_head, sizeof(char), length_read-length_head, fp); 
    	}

   	 	while(length_read < length_response)
    	{
    		int length = read(sockfd, response, RESPONSE_SIZE);
    		length_read += length;
//printf("response:\n%s\n", response);
    		//char * content = strstr(response, "\r\n\r\n"); //截获返回头部，以\r\n\r\n为标识

    		if(!strcmp(mode, "ab+"))
    		{
    			fwrite(response+length_head, sizeof(char), length-length_head, fp);
    		}
    		else
    		{
    			ungzip(response+length_head, length-length_head, fp);
    				//fwrite(response+length_head, sizeof(char), length-length_head, fp);
    		}
    	}
    }
    else
    {//puts("inside!");
	 //fflush(stdout);
    		char *content = strstr(response, "\r\n\r\n"); //截获返回头部，以\r\n\r\n为标识
    		int length_head = content-response+4;   		
    		//char *content = strstr(response+length_head, "\r\n");
    		char **endptr, *p;
    		//content = strstr(response+length_head, "\r\n");
    		char hex[10];
    		int i = 0;

    		for(p = content+4; *p != '\r'; ++p)
    		{
    			hex[i++] = *p;
    		}

    		hex[i] = '\0';
    		/*unsigned long length = 0;

    		printf("hex = %s, strlen(hex) = %d, strtoul = %lu\n", hex, strlen(hex), strtoul(hex, endptr, 16));*/
    		//strtoul()将endptr作为参数传入会引起段错误
    		while(strtoul(hex, 0, 16))
    		{
    			content = strstr(response+length_head, "\r\n");
    			length_head = content-response+2;

    			if(!strcmp(mode, "ab+"))
    			{
    				fwrite(response+length_head, sizeof(char), length_read-length_head, fp);
    			}
    			else
    			{
    				//ungzip(response+length_head, length_read-length_head, fp);
    				fwrite(response+length_head, sizeof(char), length_read-length_head, fp);
    			}

    			length_read = read(sockfd, response, RESPONSE_SIZE);
    			//content = strstr(response, "\r\n");
    			for(i = 0, p = response; *p != '\r'; ++p)
    			{
    				hex[i++] = *p;
    			}

    			hex[i] = '\0';
    		}
    }

    fcloseall();
}